{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "per_DQN.ipynb",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Q_2JwUaMheL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import random\n",
        "import numpy\n",
        "from collections import deque\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.autograd as autograd\n",
        "import numpy as np"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AGIMW2KONnGx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def mini_batch_train(env, agent, max_episodes, max_steps, batch_size):\n",
        "    episode_rewards = []\n",
        "\n",
        "    for episode in range(max_episodes):\n",
        "        state = env.reset()\n",
        "        episode_reward = 0\n",
        "\n",
        "        for step in range(max_steps):\n",
        "            action = agent.get_action(state)\n",
        "            next_state, reward, done, _ = env.step(action)\n",
        "            agent.replay_buffer.push(state, action, reward, next_state, done)\n",
        "            episode_reward += reward\n",
        "\n",
        "            if len(agent.replay_buffer) > batch_size:\n",
        "                agent.update(batch_size)   \n",
        "\n",
        "            if done or step == max_steps-1:\n",
        "                episode_rewards.append(episode_reward)\n",
        "                print(\"Episode \" + str(episode) + \": \" + str(episode_reward))\n",
        "                break\n",
        "\n",
        "            state = next_state\n",
        "\n",
        "    return episode_rewards"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k850_gIqMU2X",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class SumTree:\n",
        "    write = 0\n",
        "\n",
        "    def __init__(self, capacity):\n",
        "        self.capacity = capacity\n",
        "        self.tree = np.zeros( 2*capacity - 1 )\n",
        "        self.data = np.zeros( capacity, dtype=object )\n",
        "\n",
        "    def _propagate(self, idx, change):\n",
        "        parent = (idx - 1) // 2\n",
        "\n",
        "        self.tree[parent] += change\n",
        "\n",
        "        if parent != 0:\n",
        "            self._propagate(parent, change)\n",
        "\n",
        "    def _retrieve(self, idx, s):\n",
        "        left = 2 * idx + 1\n",
        "        right = left + 1\n",
        "\n",
        "        if left >= len(self.tree):\n",
        "            return idx\n",
        "\n",
        "        if s <= self.tree[left]:\n",
        "            return self._retrieve(left, s)\n",
        "        else:\n",
        "            return self._retrieve(right, s-self.tree[left])\n",
        "\n",
        "    def total(self):\n",
        "        return self.tree[0]\n",
        "\n",
        "    def add(self, p, data):\n",
        "        idx = self.write + self.capacity - 1\n",
        "\n",
        "        self.data[self.write] = data\n",
        "        self.update(idx, p)\n",
        "\n",
        "        self.write += 1\n",
        "        if self.write >= self.capacity:\n",
        "            self.write = 0\n",
        "\n",
        "    def update(self, idx, p):\n",
        "        change = p - self.tree[idx]\n",
        "\n",
        "        self.tree[idx] = p\n",
        "        self._propagate(idx, change)\n",
        "\n",
        "    def get(self, s):\n",
        "        idx = self._retrieve(0, s)\n",
        "        dataIdx = idx - self.capacity + 1\n",
        "\n",
        "        return (idx, self.tree[idx], self.data[dataIdx])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_oZkUpHMa32",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PrioritizedBuffer:\n",
        "\n",
        "    def __init__(self, max_size, alpha=0.6, beta=0.4):\n",
        "        self.sum_tree = SumTree(max_size)\n",
        "        self.alpha = alpha\n",
        "        self.beta = beta\n",
        "        self.current_length = 0\n",
        "\n",
        "    def push(self, state, action, reward, next_state, done):\n",
        "        priority = 1.0 if self.current_length is 0 else self.sum_tree.tree.max()\n",
        "        self.current_length = self.current_length + 1\n",
        "        #priority = td_error ** self.alpha\n",
        "        experience = (state, action, np.array([reward]), next_state, done)\n",
        "        self.sum_tree.add(priority, experience)\n",
        "\n",
        "    def sample(self, batch_size):\n",
        "        batch_idx, batch, IS_weights = [], [], []\n",
        "        segment = self.sum_tree.total() / batch_size\n",
        "        p_sum = self.sum_tree.tree[0]\n",
        "\n",
        "        for i in range(batch_size):\n",
        "            a = segment * i\n",
        "            b = segment * (i + 1)\n",
        "\n",
        "            s = random.uniform(a, b)\n",
        "            idx, p, data = self.sum_tree.get(s)\n",
        "\n",
        "            batch_idx.append(idx)\n",
        "            batch.append(data)\n",
        "            prob = p / p_sum\n",
        "            IS_weight = (self.sum_tree.total() * prob) ** (-self.beta)\n",
        "            IS_weights.append(IS_weight)\n",
        "\n",
        "        state_batch = []\n",
        "        action_batch = []\n",
        "        reward_batch = []\n",
        "        next_state_batch = []\n",
        "        done_batch = []\n",
        "\n",
        "        for transition in batch:\n",
        "            state, action, reward, next_state, done = transition\n",
        "            state_batch.append(state)\n",
        "            action_batch.append(action)\n",
        "            reward_batch.append(reward)\n",
        "            next_state_batch.append(next_state)\n",
        "            done_batch.append(done)\n",
        "\n",
        "        return (state_batch, action_batch, reward_batch, next_state_batch, done_batch), batch_idx, IS_weights\n",
        "\n",
        "    def update_priority(self, idx, td_error):\n",
        "        priority = td_error ** self.alpha\n",
        "        self.sum_tree.update(idx, priority)\n",
        "\n",
        "    def __len__(self):\n",
        "        return self.current_length"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rMys-IZuMo9p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class ConvDQN(nn.Module):\n",
        "    \n",
        "    def __init__(self, input_dim, output_dim):\n",
        "        super(ConvDQN, self).__init__()\n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        self.fc_input_dim = self.feature_size()\n",
        "        \n",
        "        self.conv = nn.Sequential(\n",
        "            nn.Conv2d(self.input_dim[0], 32, kernel_size=8, stride=4),\n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(32, 64, kernel_size=4, stride=2),\n",
        "            nn.ReLU(),\n",
        "            nn.Conv2d(64, 64, kernel_size=3, stride=1),\n",
        "            nn.ReLU()\n",
        "        )\n",
        "\n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(self.fc_input_dim, 128),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(128, 256),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(256, self.output_dim)\n",
        "        )\n",
        "\n",
        "    def forward(self, state):\n",
        "        features = self.conv_net(state)\n",
        "        features = features.view(features.size(0), -1)\n",
        "        qvals = self.fc(features)\n",
        "        return qvals\n",
        "\n",
        "    def feature_size(self):\n",
        "        return self.conv_net(autograd.Variable(torch.zeros(1, *self.input_dim))).view(1, -1).size(1)\n",
        "\n",
        "\n",
        "class DQN(nn.Module):\n",
        "    \n",
        "    def __init__(self, input_dim, output_dim):\n",
        "        super(DQN, self).__init__()\n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(self.input_dim[0], 128),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(128, 256),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(256, self.output_dim)\n",
        "        )\n",
        "\n",
        "    def forward(self, state):\n",
        "        qvals = self.fc(state)\n",
        "        return qvals"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BhLjRHaKMqxa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class PERAgent:\n",
        "\n",
        "    def __init__(self, env, use_conv=True, learning_rate=3e-4, gamma=0.99, buffer_size=10000):\n",
        "        self.env = env\n",
        "        self.gamma = gamma\n",
        "        self.replay_buffer = PrioritizedBuffer(buffer_size)\n",
        "        self.device = \"cpu\"\n",
        "        if torch.cuda.is_available():\n",
        "            self.device = \"cuda\"\n",
        "        \n",
        "        if use_conv:\n",
        "            self.model = ConvDQN(self.env.observation_space.shape, env.action_space.n).to(self.device)\n",
        "        else:\n",
        "            self.model = DQN(self.env.observation_space.shape, env.action_space.n).to(self.device)\n",
        "          \n",
        "        self.optimizer = torch.optim.Adam(self.model.parameters())\n",
        "        self.MSE_loss = nn.MSELoss()\n",
        "\n",
        "    def get_action(self, state, eps=0.0):\n",
        "        state = torch.FloatTensor(state).float().unsqueeze(0).to(self.device)\n",
        "        qvals = self.model.forward(state)\n",
        "        action = np.argmax(qvals.cpu().detach().numpy())\n",
        "        \n",
        "        if(np.random.rand() > eps):\n",
        "            return self.env.action_space.sample()\n",
        "          \n",
        "        return action\n",
        "\n",
        "    def _sample(self, batch_size):\n",
        "        return self.replay_buffer.sample(batch_size)\n",
        "\n",
        "    def _compute_TDerror(self, batch_size):\n",
        "        transitions, idxs, IS_weights = self._sample(batch_size)\n",
        "        states, actions, rewards, next_states, dones = transitions\n",
        "\n",
        "        states = torch.FloatTensor(states).to(self.device)\n",
        "        actions = torch.LongTensor(actions).to(self.device)\n",
        "        rewards = torch.FloatTensor(rewards).to(self.device)\n",
        "        next_states = torch.FloatTensor(next_states).to(self.device)\n",
        "        dones = torch.FloatTensor(dones).to(self.device)\n",
        "        IS_weights = torch.FloatTensor(IS_weights).to(self.device)\n",
        "\n",
        "        curr_Q = self.model.forward(states).gather(1, actions.unsqueeze(1))\n",
        "        curr_Q = curr_Q.squeeze(1)\n",
        "        next_Q = self.model.forward(next_states)\n",
        "        max_next_Q = torch.max(next_Q, 1)[0]\n",
        "        expected_Q = rewards.squeeze(1) + self.gamma * max_next_Q\n",
        "\n",
        "        td_errors = torch.pow(curr_Q - expected_Q, 2) * IS_weights\n",
        "\n",
        "        return td_errors, idxs\n",
        "\n",
        "    def update(self, batch_size):\n",
        "        td_errors, idxs = self._compute_TDerror(batch_size)\n",
        "\n",
        "        # update model\n",
        "        td_errors_mean = td_errors.mean()\n",
        "        self.optimizer.zero_grad()\n",
        "        td_errors_mean.backward()\n",
        "        self.optimizer.step()\n",
        "\n",
        "        # update priorities\n",
        "        for idx, td_error in zip(idxs, td_errors.cpu().detach().numpy()):\n",
        "            self.replay_buffer.update_priority(idx, td_error)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pdI6we76NdhA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "f0507660-27ca-4f21-d1f9-7918dbb73367"
      },
      "source": [
        "import gym\n",
        "\n",
        "env_id = \"CartPole-v0\"\n",
        "MAX_EPISODES = 1000\n",
        "MAX_STEPS = 500\n",
        "BATCH_SIZE = 32\n",
        "\n",
        "env = gym.make(env_id)\n",
        "agent = PERAgent(env, use_conv=False)\n",
        "episode_rewards = mini_batch_train(env, agent, MAX_EPISODES, MAX_STEPS, BATCH_SIZE)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Episode 0: 31.0\n",
            "Episode 1: 13.0\n",
            "Episode 2: 16.0\n",
            "Episode 3: 13.0\n",
            "Episode 4: 34.0\n",
            "Episode 5: 32.0\n",
            "Episode 6: 31.0\n",
            "Episode 7: 98.0\n",
            "Episode 8: 21.0\n",
            "Episode 9: 29.0\n",
            "Episode 10: 11.0\n",
            "Episode 11: 33.0\n",
            "Episode 12: 21.0\n",
            "Episode 13: 27.0\n",
            "Episode 14: 17.0\n",
            "Episode 15: 18.0\n",
            "Episode 16: 22.0\n",
            "Episode 17: 12.0\n",
            "Episode 18: 14.0\n",
            "Episode 19: 43.0\n",
            "Episode 20: 21.0\n",
            "Episode 21: 68.0\n",
            "Episode 22: 20.0\n",
            "Episode 23: 18.0\n",
            "Episode 24: 30.0\n",
            "Episode 25: 11.0\n",
            "Episode 26: 29.0\n",
            "Episode 27: 11.0\n",
            "Episode 28: 32.0\n",
            "Episode 29: 33.0\n",
            "Episode 30: 12.0\n",
            "Episode 31: 24.0\n",
            "Episode 32: 23.0\n",
            "Episode 33: 14.0\n",
            "Episode 34: 20.0\n",
            "Episode 35: 23.0\n",
            "Episode 36: 15.0\n",
            "Episode 37: 23.0\n",
            "Episode 38: 34.0\n",
            "Episode 39: 24.0\n",
            "Episode 40: 16.0\n",
            "Episode 41: 27.0\n",
            "Episode 42: 20.0\n",
            "Episode 43: 12.0\n",
            "Episode 44: 30.0\n",
            "Episode 45: 19.0\n",
            "Episode 46: 18.0\n",
            "Episode 47: 19.0\n",
            "Episode 48: 19.0\n",
            "Episode 49: 24.0\n",
            "Episode 50: 19.0\n",
            "Episode 51: 22.0\n",
            "Episode 52: 22.0\n",
            "Episode 53: 17.0\n",
            "Episode 54: 22.0\n",
            "Episode 55: 24.0\n",
            "Episode 56: 23.0\n",
            "Episode 57: 17.0\n",
            "Episode 58: 23.0\n",
            "Episode 59: 14.0\n",
            "Episode 60: 14.0\n",
            "Episode 61: 50.0\n",
            "Episode 62: 31.0\n",
            "Episode 63: 36.0\n",
            "Episode 64: 11.0\n",
            "Episode 65: 9.0\n",
            "Episode 66: 10.0\n",
            "Episode 67: 13.0\n",
            "Episode 68: 39.0\n",
            "Episode 69: 15.0\n",
            "Episode 70: 23.0\n",
            "Episode 71: 18.0\n",
            "Episode 72: 19.0\n",
            "Episode 73: 45.0\n",
            "Episode 74: 26.0\n",
            "Episode 75: 24.0\n",
            "Episode 76: 40.0\n",
            "Episode 77: 26.0\n",
            "Episode 78: 26.0\n",
            "Episode 79: 18.0\n",
            "Episode 80: 12.0\n",
            "Episode 81: 22.0\n",
            "Episode 82: 22.0\n",
            "Episode 83: 24.0\n",
            "Episode 84: 12.0\n",
            "Episode 85: 10.0\n",
            "Episode 86: 16.0\n",
            "Episode 87: 13.0\n",
            "Episode 88: 21.0\n",
            "Episode 89: 16.0\n",
            "Episode 90: 29.0\n",
            "Episode 91: 25.0\n",
            "Episode 92: 10.0\n",
            "Episode 93: 10.0\n",
            "Episode 94: 17.0\n",
            "Episode 95: 24.0\n",
            "Episode 96: 28.0\n",
            "Episode 97: 19.0\n",
            "Episode 98: 20.0\n",
            "Episode 99: 12.0\n",
            "Episode 100: 29.0\n",
            "Episode 101: 21.0\n",
            "Episode 102: 9.0\n",
            "Episode 103: 21.0\n",
            "Episode 104: 18.0\n",
            "Episode 105: 15.0\n",
            "Episode 106: 23.0\n",
            "Episode 107: 21.0\n",
            "Episode 108: 12.0\n",
            "Episode 109: 12.0\n",
            "Episode 110: 22.0\n",
            "Episode 111: 31.0\n",
            "Episode 112: 15.0\n",
            "Episode 113: 10.0\n",
            "Episode 114: 13.0\n",
            "Episode 115: 58.0\n",
            "Episode 116: 16.0\n",
            "Episode 117: 12.0\n",
            "Episode 118: 31.0\n",
            "Episode 119: 48.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-fae0016bae11>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0menv\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgym\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv_id\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0magent\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPERAgent\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0muse_conv\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0mepisode_rewards\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmini_batch_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0menv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0magent\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMAX_EPISODES\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mMAX_STEPS\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mBATCH_SIZE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-2-1d2ff31503ef>\u001b[0m in \u001b[0;36mmini_batch_train\u001b[0;34m(env, agent, max_episodes, max_steps, batch_size)\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplay_buffer\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m                 \u001b[0magent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mmax_steps\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-10-3121e77d732e>\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, batch_size)\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0mtd_errors_mean\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtd_errors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 59\u001b[0;31m         \u001b[0mtd_errors_mean\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     60\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     61\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    100\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    101\u001b[0m         \"\"\"\n\u001b[0;32m--> 102\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    103\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    104\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     88\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     89\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 90\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     91\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rFabyqT_Nk1O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}